'''
@Author: YSZ
@Date: 2019-12-17 13:13:53
@LastEditors  : ysz
@LastEditTime : 2020-01-13 17:48:43
@Description: 爬取58同城二手房的个人房源的房源基本信息
'''
import json
import html
import requests
import re
import os.path
import os
import time
import datetime
import random
from bs4 import BeautifulSoup
from lxml import etree

from jiemi_help import Jiemi
from mysqlhelp import MysqlHelper
from user_agent_help import User_Agent_Help
from common_object import TongCheng

#实例化解码类的对象
jm_help=Jiemi()
mysqlhelp=MysqlHelper()
uahelp=User_Agent_Help()
user_agent=uahelp.get_useragent()

class Secondary_House(TongCheng):

    # ----获取列表页中关于详情页的url
    def page_list(self,url,referer):
        '''
        @description: 根据指定的列表页，获取在列表中的详情页的url
        @param1: {
        *      self:"当前对象",
        *      url:"列表页的url",
        *      referer:"从哪个页面获取到url的",
        *      }
        @return: 无
        @detail: 根据xpath获取详情页的url，
                 同时根据获取的详情页的url，组成列表，
                 循环执行获取详情页内容的函数
        '''
        list_html=super().page_list(url,referer)
        #获取当前列表页中的详情页的url列表
        detail_url_list=list_html.xpath('//div[@class="list-info"]/h2/a/@href')
        #遍历url列表，获取详情页中的内容
        for detail_url in detail_url_list:
            if 'ershoufang' in detail_url:
                time.sleep(random.randint(60,120))
                self.one_detail_page(detail_url,url)

        
    #爬取详情页的数据
    def one_detail_page(self,url,referer):
        '''
        @description: 获取详情页的内容
        @param1: {
        *      self:"当前对象",
        *      url:"当前详情页的url",
        *      referer:"从哪个页面中获取当前url的",
        *      }
        @return: 无
        @detail: 根据xpath获取网页中指定数据，并存储到数据库
        '''
        print(url,'详情页的地址')
        # global user_agent
        headers={
            'cookie':self.headers['cookie'],
            'User-Agent':self.headers['User-Agent'],
            'referer': referer,
        }
        res=requests.get(url,headers=headers)
        rest=etree.HTML(res.text)
        if '你要找的页面不在这个星球上！' not in rest.xpath('//h1/text()')[0].strip():
            #将&#x958f;室变为閏室
            text = html.unescape(res.text)  
            #用正则表达式提取AAE..AAA
            key = re.findall(r"base64,(.*)'\).format", text)[0]  

            # 获取转换编码格式后的网页原码
            dehtml = jm_help.decode58Fangchan(text, key)
            page = etree.HTML(dehtml)
       
            #获取页面信息
            #标题
            title=page.xpath('//h1[@class="c_333 f20"]/text()')[0].strip()

            #概况
            #房屋总价
            moneyTags=page.xpath('//span[@class="c_000 strongbox"]/text()')[0].strip()
            total=moneyTags.split('(')[0].replace('万','')
            unit=moneyTags.split('(')[-1].replace('单价','').replace('元/㎡)','')
            #房屋类型
            room_model=page.xpath('//p[@class="room"]/span[@class="main"]/text()')[0].strip()
            model1=room_model[0]
            model2=room_model[2]
            model3=room_model[4]
            #房屋面积
            area=page.xpath('//p[@class="area"]/span[@class="main"]/text()')[0].strip().replace('平','')
            #房屋朝向
            direction=page.xpath('//p[@class="toward"]/span[@class="main"]/text()')[0].strip()
            #所属小区
            if page.xpath('//ul[@class="house-basic-item3"]/li[1]/span[@class="c_000 mr_10"][1]//a/text()'):
                community=page.xpath('//ul[@class="house-basic-item3"]/li[1]/span[@class="c_000 mr_10"][1]//a/text()')
            else:
                community=page.xpath('//ul[@class="house-basic-item3"]/li[1]/span[@class="c_000 mr_10"][1]/text()')
            xq=community[0].strip()

            #具体位置
            location=page.xpath('//ul[@class="house-basic-item3"]/li[2]/span[@class="c_000 mr_10"][1]//a/text()')
            wz=location[0].strip()

            
            #获取主要数据来源
            person_name=re.findall('"userName":"(.*?)",',dehtml,re.S)[0].strip().encode("utf-8").decode("unicode_escape")
            #联系人电话
            phone=page.xpath('//p[@class="phone-num"]/text()')[0].strip()

            #所在楼层
            co_floor=page.xpath('//ul[@class="general-item-right"][1]/li[1]/span[2]/text()')[0].strip()
            total_layer=re.findall('\d+',co_floor)[0].strip()
            floor=co_floor.split('(')[0].strip()
            floor=self.judge_scope(floor,int(total_layer))
            #房屋装修状况
            decorate=page.xpath('//ul[@class="general-item-right"][1]/li[2]/span[2]/text()')[0]
            if decorate=='毛坯':
                isdecorate=0
            else:
                isdecorate=1
            #产权年限
            limit_year=page.xpath('//ul[@class="general-item-right"][1]/li[3]/span[2]/text()')[0].replace('年','').strip()
            if  page.xpath('//ul[@class="general-item-right"][1]/li[4]/span[2]/text()'):
                nj=page.xpath('//ul[@class="general-item-right"][1]/li[4]/span[2]/text()')[0].strip().replace('年','')
            else:
                nj=0
            
            #房屋描述
            #  房屋简介
            desc=page.xpath('//p[@class="pic-desc-word"]/text()')[0].strip()
            #费用
            # 参考首付
            refer_money=page.xpath('//div[@class="general-item general-expense"]//ul[@class="general-item-right"]/li/span[2]/text()')[0].strip()
            sf=refer_money.split('（')[0].replace('万','').strip()
            yg=refer_money.split('（')[1]
            yg=re.findall('\d+',yg)[0].strip()
            # 房屋类型
            house_type=page.xpath('//div[@class="general-item general-expense"]//ul[@class="general-item-left"]/li[2]/span[2]/text()')[0].strip()
            # 交易权属
            trading_ownership=page.xpath('//div[@class="general-item general-expense"]//ul[@class="general-item-left"]/li[3]/span[2]/text()')[0].strip()


            imglist=page.xpath('//ul[@class="general-pic-list"]/li/img/@data-src')
            # imglist=[re.sub('/',"\/",i) for i in imglist]
            imgs=json.dumps(imglist)
            print(imgs)
            
            
            #房源的url的id
            about_id=url.split('?')[0].split('/')[-1].split('.')[0].replace('x','')
            # 房源信息的来源在58同城上的url地址
            about_url=url
            #爬取的时间
            create_time=int(time.time())
            #状态
            status=1
            print(mysqlhelp.query_data(about_id=about_id,area=area,xq=xq,phone=phone,house_type='secondary'),'能不能往数据库存储')
            if mysqlhelp.query_data(about_id=about_id,area=area,xq=xq,phone=phone,house_type='secondary'):
                sql='insert into db_housesecond_sh(about_id,about_url,title,model1,model2,model3,area,total,unit,direction,layer,total_layer,isdecorate,decorate,`desc`,xq,wz,`status`,create_time,imgs,owner_name,owner_phone,sf,yg,zyqs,`type`,ncq,nj) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
                data=[about_id,about_url,title,model1,model2,model3,area,total,unit,direction,floor,total_layer,isdecorate,decorate,desc,xq,wz,status,create_time,imgs,person_name,phone,sf,yg,trading_ownership,house_type,limit_year,nj]
                house_id=mysqlhelp.sql_exec(sql,data)
                print(house_id,'房源id')
                # print('开始下载图片--------------------------------------------------------')
                # print(imglist,'图片url列表',type(imglist))
                # for img_url in imglist:
                #     create_time=int(time.time())
                #     pic_path=self.down_img_pic(img_url,url,house_type='secondary')
                #     sql='insert into db_picture2(path,status,create_time,housesecond_id) values(%s,%s,%s,%s)'
                #     data=[pic_path,1,create_time,house_id]
                #     mysqlhelp.sql_exec(sql,data)
                # print('图片下载完成')
            # print('本次循环的信息：')
            # print('-----'*20)
            # #字段设置
            # print(about_id,'房源id')
            # print(about_url,'房源url')
            print(title,'标题')#--
            # print(model1,'室')
            # print(model2,'厅')
            # print(model3,'卫')
            # print(area,'面积')
            # print(total,'总价')
            # print(unit,'单价')
            # print(direction,'房屋朝向')
            # print(floor,'楼层')
            # print(total_layer,'总楼层')
            # print(isdecorate,'是否装修')
            # print(decorate,'装修')
            # print(desc,'房屋描述')
            # print(xq,'小区')
            # print(wz,'位置')
            # print(status,'房屋状态')
            # print(create_time,'爬取房屋时间戳')
            # print(imglist,'图片url列表')
            # print(person_name,'业主姓氏')
            # print(phone,'业主电话')
            # print(sf,'房屋首付金额')
            # print(yg,'月供')
            # print(trading_ownership,'交易权属')
            # print(house_type,'房屋类型')
            # print(limit_year,'年产权')
            # print(nj,'年建')
            print('本页面爬取完毕-------------------------------------------------------')
            print()
        else:
            print('没有信息')




# 代码的入口
if __name__ == "__main__":
    # path='./download'
    # if not os.path.exists(path):
    #     os.mkdir(path)
    url='https://sy.58.com/ershoufang/0/pn1/'
    headers={
        'cookie':'f=n; commontopbar_new_city_info=188%7C%E6%B2%88%E9%98%B3%7Csy; f=n; commontopbar_new_city_info=188%7C%E6%B2%88%E9%98%B3%7Csy; time_create=1579137529481; userid360_xml=95ACC2EDA5D9DF3D3463242427FA5BFD; f=n; commontopbar_new_city_info=188%7C%E6%B2%88%E9%98%B3%7Csy; id58=e87rZl3mC5iLtxTfA0qZAg==; wmda_uuid=83b369d94097739f0fd218170fae2922; wmda_new_uuid=1; 58tj_uuid=907d287a-a241-4d7f-b4a9-82f5dc178ef4; als=0; xxzl_deviceid=Li4jM92s0F9MJHnfmW1z1NQ3xUQFJVP94fV2NomSldd3n%2B795%2BBDxO9iDZCQMdlN; Hm_lvt_ae019ebe194212c4486d09f377276a77=1575357349; 58home=sy; myLat=""; myLon=""; mcity=sy; ctid=39; aQQ_ajkguid=1737BD87-E303-12D4-8D33-SX1217091934; __xsptplus8=8.1.1576545583.1576545632.2%234%7C%7C%7C%7C%7C%23%234UPLQ5z-MySS0MJxtXGvRal3qitr3VFH%23; myfeet_tooltip=end; m58comvp=t18v115.159.229.14; Hm_lvt_295da9254bbc2518107d846e1641908e=1576646576; city=sy; param8616=1; param8716kop=1; Hm_lvt_a3013634de7e7a5d307653e15a0584cf=1577260386; wmda_visited_projects=%3B6333604277682%3B11187958619315%3B1731916484865; f=n; commontopbar_new_city_info=188%7C%E6%B2%88%E9%98%B3%7Csy; commontopbar_ipcity=sy%7C%E6%B2%88%E9%98%B3%7C0; xxzl_cid=629caa9d45574456b4659ab0481de6ea; xzuid=33e294c9-8204-4c58-a9f6-1fa708893b6c; defraudName=defraud; ppStore_fingerprint=37C4AA4FC5419C853B885C093FB1B48F92BDE760BC6D8D01%EF%BC%BF1577408663372; wmda_session_id_6333604277682=1577411697662-68c244c7-c7e5-17a8; new_uv=6; utm_source=; spm=; init_refer=https%253A%252F%252Fsy.58.com%252F; wmda_session_id_11187958619315=1577411733439-80453495-f2fe-fde1; new_session=0; JSESSIONID=0B379D04EB0A52F2D41C3A4A3D01CE13; xzfzqtoken=G1eAQvcVAJNGq%2BK31K0IXZBWecSWOAC0fYZbQgHU9yseCySfpiU7nz7Z6lBzbUAMin35brBb%2F%2FeSODvMgkQULA%3D%3D',
        'referer':'https://callback.58.com/antibot/verifycode?serialId=409f05bcbeeeff066c7091eb17b5fde9_f3ebe93edc1d424eaef31f731acf7403&code=22&sign=a62b68976cfa48a25f02db9b8797c8b8&namespace=ershoufanglistphp&url=https%3A%2F%2Fsy.58.com%2Fershoufang%2F0%2Fpn2%2F',
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
    }
    mysecond=Secondary_House(url,headers)
    mysecond.run()
    # url='https://sy.58.com/ershoufang/40551377428645x.shtml?spm=&cityListName=sy&curListName=sy&referinfo=FALSE&typecode=203&from=1-list-1&iuType=z_0&PGTID=0d30000c-000b-c9c5-7a7a-312fa68e7b4b&ClickID=1&adtype=3'
    # mysecond.one_detail_page(url,'https://sy.58.com/ershoufang/0/pn1/')

    
   
    





